package cn.edu.tsinghua.xkk.retrieval;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectOutputStream;
import java.io.Serializable;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.Tag;
import org.htmlparser.Text;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

import cn.edu.tsinghua.xkk.entity.*;

/**
 * analyze the stored page
 * @author shibei
 *
 */
public class XKKAnalizer implements Serializable{
	
	static final long serialVersionUID = 123456L;
	public String parentPath;
	public XKKInvertedIndex invertedIndex = new XKKInvertedIndex();
	public static Map<String, Integer> tagMap = new HashMap();
	
	static
	{
		tagMap.put("title", 40);
		tagMap.put("cite", 8);
		tagMap.put("em", 2);
		tagMap.put("strong", 4);
		tagMap.put("b", 4);
		tagMap.put("i", 2);
		tagMap.put("big", 4);
		tagMap.put("h1", 12);
		tagMap.put("h2", 8);
		tagMap.put("h3", 1);
		tagMap.put("h4", 1);
		tagMap.put("dt", 4);
		tagMap.put("ul", 4);
		tagMap.put("li", 4);
		tagMap.put("font size=7", 16);
		tagMap.put("font size=6", 12);
		tagMap.put("font size=5", 8);
		tagMap.put("font size=4", 4);
		tagMap.put("font size=3", 1);
		tagMap.put("font size=2", 1);
		tagMap.put("font size=1", 1);
	}

	public XKKAnalizer(String parentPath)
	{
		this.parentPath = parentPath;
	}
	
	public XKKAnalizer()
	{

	}
	
	public void analyze()
	{
		analyze(this.parentPath);
	}
	
	public void analyze(String s)
	{
		File tempParentFile = new File(s);
		File[] allFile = tempParentFile.listFiles();
		
		if(allFile == null)
		{
			return;
		}
		else
		{
			for(int i = 0; i < 2046; i++)
			{
//				if(i % 1000 == 0)
//				{
//					ObjectOutputStream f;
//					try {
//						f = new ObjectOutputStream(new FileOutputStream("Inverted.dat"));
//						f.writeObject(invertedIndex);
//						f.close();
//						System.out.println(this.parentPath +"\\" +  i +"  has written!");
//					} catch (FileNotFoundException e) {
//						// TODO Auto-generated catch block
//						e.printStackTrace();
//					} catch (IOException e) {
//						// TODO Auto-generated catch block
//						e.printStackTrace();
//					}
//				}
				
				XKKPage p = readXKKFile(new File(this.parentPath + "\\" + i));
				
				if(p == null)
				{
					continue;
				}
				
				if(p.getConnection() == true)
				{
					List<XKKNode> result = getTokenFromPage(p);
					addInvertedIndex(result, p.getID());
					System.out.println(this.parentPath + "\\" + i + " completed!");
				}
//				System.gc();
			}
			try {
				ObjectOutputStream f = new ObjectOutputStream(new FileOutputStream("Inverted.dat"));
				f.writeObject(invertedIndex);
				f.close();
				System.out.println("All has written!");
			} catch (FileNotFoundException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
		}
	}
	
	private void addInvertedIndex(List<XKKNode> listToken, int id) {
		// TODO Auto-generated method stub
		Iterator<XKKNode> it = listToken.iterator();
		Collections.sort(listToken);
		String lastString = "";
		while(it.hasNext())
		{
			XKKNode temp = (XKKNode) it.next();
			if(lastString.compareTo(temp.content) == 0)
			{
				List<XKKIndexNode> tempList = invertedIndex.index.get(temp.content);
				tempList.get(tempList.size() - 1).weight += temp.weight;
				continue;
			}
			else
			{
				if(invertedIndex.index.containsKey(temp.content))
				{
					List<XKKIndexNode> tempList = invertedIndex.index.get(temp.content);
					tempList.add(new XKKIndexNode(id, temp.weight));
				}
				else
				{
					List<XKKIndexNode> tempList = new ArrayList<XKKIndexNode>();
					tempList.add(new XKKIndexNode(id, temp.weight));
					invertedIndex.index.put(temp.content, tempList);
				}
				lastString = temp.content;
			}
		}
	}

	private List<XKKNode> getTokenFromPage(XKKPage p) {
		// TODO Auto-generated method stub
	
		List<XKKNode> result = new ArrayList<XKKNode>();
		try {
			Parser parser = Parser.createParser(p.getAllContent(), p.getEncoding());
			NodeIterator ni = parser.elements();
			while(ni.hasMoreNodes())
			{
				Node tempNode = ni.nextNode();
				dealNode(tempNode, result, p);
			}
		} catch (ParserException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		catch(Exception e)
		{
			e.printStackTrace();
		}
		return result;
	}

	private void dealNode(Node n, List<XKKNode> tl, XKKPage p)
	{
		if(n instanceof Tag)
		{
			Tag tempT = (Tag)n;
			NodeList nl = tempT.getChildren();
			if(nl != null)
			{
				NodeIterator ni = nl.elements();
				try {
					while(ni.hasMoreNodes())
					{
						Node tempN = ni.nextNode();
						dealNode(tempN, tl, p);
					}
				} catch (ParserException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				}
			}	
		}
		else if(n instanceof Text)
		{
			String s = n.getText();
			s = s.replaceAll("\n", " ");
			s = s.replaceAll("\r", " ");
			s = s.trim();
			
			if(s.compareTo("") != 0)
			{
				int weight = 0;
				List<String> tempStringList = XKKChineseSegment.segment(s);
				if(n.getParent() != null)
				{
					Node parent = n.getParent();
					if(parent instanceof Tag)
					{
						Tag t = (Tag)parent;
						String tagname = t.getRawTagName();
						if(tagMap.containsKey(tagname))
						{
							weight = tagMap.get(tagname);
						}
						else
						{
							weight = 2;
						}
					}
					else
					{
						weight = 1;
					}
				}
				else
				{
					weight = 1;
				}
				List<XKKNode> tempXKKNodeList = new ArrayList();
				Iterator<String> stringIt = tempStringList.iterator();
				while(stringIt.hasNext())
				{
					tempXKKNodeList.add(new XKKNode(stringIt.next(), weight));
				}
				tl.addAll(tempXKKNodeList);
			}
		}
	}
	
	public XKKPage readXKKFile(File f)
	{
		XKKPage p = new XKKPage();
		if(!f.exists())
			return null;
		try {
			BufferedReader br = new BufferedReader(new FileReader(f));
			String tempLine = "";
			String currentLine = "";
			while((tempLine = br.readLine()) != null)
			{
				if(tempLine.startsWith("#ID:"))
				{
					p.setID(Integer.valueOf(br.readLine()));
				}
				else if(tempLine.startsWith("#URL:"))
				{
					p.setURL(br.readLine());
				}
				else if(tempLine.startsWith("#Title:"))
				{
					p.setTitle(br.readLine());
				}
				else if(tempLine.startsWith("#Type:"))
				{
					p.setType(br.readLine());
				}
				else if(tempLine.startsWith("#Connection flag:"))
				{
					p.setConnection(Boolean.valueOf(br.readLine()));
				}
				else if(tempLine.startsWith("#Charset:"))
				{
					p.setEncoding(br.readLine());
				}
				else if(tempLine.startsWith("#All content:"))
				{
					currentLine = "";
				}
				else if(tempLine.startsWith("#Children links:"))
				{
					p.setAllContent(currentLine);
					currentLine = "";
				}
				else
				{
					currentLine += tempLine + "\n";
				}
			}
			br.close();
			String[] temp = currentLine.split("\n");
			for(int i = 0; i < temp.length; i++)
			{
				p.linkChildren.add(Integer.valueOf(temp[i]));
			}
		} catch (FileNotFoundException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return p;
	}
}
